package com.dtkavin.spark.demo

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by IntelliJ IDEA.
  * Programmer : John Zn
  * Date : 2016/4/11 0011
  * Time : 21:42
  * Discribtion : through logs from station and userRecord ,we can get locations of home and company
  */
class UsrPassBy {

}


object UsrPassBy {
  def main(args: Array[String]) {
    var conf = new SparkConf().setAppName("UsrPassBy").setMaster("local")
    var sc = new SparkContext(conf)

    val station = sc.textFile("D:\\data\\input-position\\station.txt").filter(!_.isEmpty).map(_.trim).map((_.split(","))).map(arr => ((arr(0), (arr(1), arr(2), arr(3)))))
    val usrInfo = sc.textFile("D:\\data\\input-position\\usr-info.txt").filter(!_.isEmpty).map(_.trim).map(_.split(",")).map(arr => ((arr(2)), (arr(0), arr(1), arr(3))))

    //      .map(arr => ((arr(0)), (arr(1), arr(2), arr(3))))
    //      .map((arr => ((arr(2)), (arr(0), arr(1), arr(3))))).sortBy(_._2)
    //    val rdd1 = sc.textFile("hdfs://spark01:9000/wordcount/a.txt")
    val bigtable = usrInfo.leftOuterJoin(station)
    bigtable.map(_._2).sortBy(_._1).foreach(println)

    sc.stop()

  }


}
